package com.lianyu507.tools;

import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;

import java.io.IOException;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

public class TokenizerTool {
    private static Analyzer analyzer = new StandardAnalyzer();

    public static List<String> tokenizer(String content)  {


        List<String> wordList = new ArrayList<>();
        if (content == null || content.length() < 1) {
            return wordList;
        }

        TokenStream tokenStream = null;
        try {
            tokenStream = analyzer.tokenStream(null, content);

            CharTermAttribute term = tokenStream.addAttribute(CharTermAttribute.class);
            tokenStream.reset();
            while (tokenStream.incrementToken()) {
                String a = term.toString();
                if (a.length() > 2 && !isNumeric(a)) {
                    wordList.add(term.toString());
                }
            }
            tokenStream.close();
        } catch (IOException e) {
            e.printStackTrace();
        }


    finally {
            try {
                tokenStream.close();
            } catch (IOException e) {
                e.printStackTrace();
            }

        }

        return removeDuplicateWithOrder(wordList);
    }

    //判断字符串是否为数字
    public static boolean isNumeric(String str) {
        Pattern pattern = Pattern.compile("[0-9]*");
        Matcher isNum = pattern.matcher(str);
        if (!isNum.matches()) {
            return false;
        }
        return true;
    }

    //单词去重
    public static List removeDuplicateWithOrder(List list) {
        Set set = new HashSet();
        List newList = new ArrayList();
        for (Iterator iter = list.iterator(); iter.hasNext(); ) {
            Object element = iter.next();
            if (set.add(element))
                newList.add(element);
        }
        list.clear();
        list.addAll(newList);
        // System.out.println( " remove duplicate "   +  list);
        return list;
    }

    //测试分词 测试之后就注释掉此方法，否则maven 编译报错，Spring boot项目只允许有一个main 方法
    
//    public static void main(String[] args) {
//        String testStr = "<p>Your Shot member Sally Nguyen's mother takes in the Vancouver waterfront while behind her a bicyclist whizzes past. Home to aboriginal people for people people people thousands of years, the port city in British Columbia, Canada, is named for the British sea captain who charted the coastline in 1792.</p>\n";
//
//        System.out.println(tokenizer(testStr));
//    }
}
